{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "jieba_test.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/henrygas/unsupervised_learning/blob/master/jieba_test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JTVV-dX6BF6_",
        "colab_type": "text"
      },
      "source": [
        "## 1. 装载Google云盘"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8-uQGxLBBmg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "9eb7adf2-0791-4c95-e4cb-70669de85888"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "77pzeIjHBWpp",
        "colab_type": "text"
      },
      "source": [
        "## 2. 定位当前工作目录"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ppX_Y2FCBZlO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "os.chdir(\"./drive/My Drive/app/jieba_test\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xCl2oemxBooN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3db7e0ce-8063-43ca-d81e-d2a905e2c7c5"
      },
      "source": [
        "!ls"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data  out\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ewib3yzGBtYe",
        "colab_type": "text"
      },
      "source": [
        "## 3. 学习jieba的使用"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "97CluWu6B0ZN",
        "colab_type": "text"
      },
      "source": [
        "### 3.1 三种分词模式的测试"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mRxh3PYBqQh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "e513b24c-f7ec-4f16-e8f1-980ad08b3497"
      },
      "source": [
        "import jieba\n",
        "\n",
        "seg_list = jieba.cut(\"我来到北京清华大学\", cut_all=True, HMM=False)\n",
        "\"Full Mode:{}\".format(\"/\".join(seg_list))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Building prefix dict from the default dictionary ...\n",
            "Dumping model to file cache /tmp/jieba.cache\n",
            "Loading model cost 0.937 seconds.\n",
            "Prefix dict has been built succesfully.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Full Mode:我/来到/北京/清华/清华大学/华大/大学'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zILD2KFpCUwo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4e5b1585-e156-44e0-928e-00cde097e6af"
      },
      "source": [
        "default_seg_list = jieba.cut(\"我来到北京清华大学\", cut_all=False, HMM=True)\n",
        "\"Default Mode:{}\".format(\"/\".join(default_seg_list))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Default Mode:我/来到/北京/清华大学'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1O-taY9UCnmY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "149a5c01-7470-4d32-8ccd-9da7767603ee"
      },
      "source": [
        "no_hmm_seg_list = jieba.cut(\"我来到北京清华大学\", cut_all=False, HMM=False)\n",
        "\"No HMM Mode:{}\".format(\"/\".join(no_hmm_seg_list))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'No HMM Mode:我/来到/北京/清华大学'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7I79PSKDC60n",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7fd84523-520e-4a7c-b08c-228b95953645"
      },
      "source": [
        "search_seg_list = jieba.cut_for_search(\"我来到北京清华大学\", HMM=False)\n",
        "\"Search Mode:{}\".format(\"/\".join(search_seg_list))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Search Mode:我/来到/北京/清华/华大/大学/清华大学'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hQ09afDsDnu8",
        "colab_type": "text"
      },
      "source": [
        "### 3.2 关键词提取"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54Y_uozGDpox",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "ce5880f2-2ee2-4945-e53c-1f6d448b2f10"
      },
      "source": [
        "from jieba import analyse\n",
        "\n",
        "content = \"我来到北京清华大学，发现清华大学的食堂比北京大学的食堂要好吃，比北京的大学都要好吃\"\n",
        "\n",
        "tags = analyse.extract_tags(content, topK=5, withWeight=True)\n",
        "for tag, weight in tags:\n",
        "  print(tag, weight)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "食堂 1.4224502728250001\n",
            "好吃 1.3497598794333332\n",
            "清华大学 1.3467657869366667\n",
            "北京 0.7779003847866667\n",
            "北京大学 0.6370773008516667\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mmVUQTiPE1YO",
        "colab_type": "text"
      },
      "source": [
        "### 3.3 中文歧义测试与去除停用词"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v16i4zBZE8BD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8b4545ff-287b-46dd-caa3-50482be2858f"
      },
      "source": [
        "import jieba\n",
        "\n",
        "content = \"2010年底了，部队的友谊篮球赛就结束了\"\n",
        "seg_list = jieba.cut(content, cut_all=True)\n",
        "\"/\".join(seg_list)\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2010/年底/了///部队/的/友谊/篮球/篮球赛/球赛/就/结束/了'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n7tt5lL3FdLt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d39fbb15-fd5b-4539-f225-273902eba131"
      },
      "source": [
        "content = \"2010年底了，部队的友谊篮球赛就结束了\"\n",
        "seg_list = jieba.cut(content, cut_all=False)\n",
        "\"/\".join(seg_list)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2010/年底/了/，/部队/的/友谊/篮球赛/就/结束/了'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-xYEaZVcFq3F",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f6d70a71-a5d9-4bc0-9cea-1573f5df77ee"
      },
      "source": [
        "content = \"2010年底了，部队的友谊篮球赛就结束了\"\n",
        "seg_list = jieba.cut_for_search(content, HMM=True)\n",
        "\"/\".join(seg_list)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2010/年底/了/，/部队/的/友谊/篮球/球赛/篮球赛/就/结束/了'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUUYcdk3F0k0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a53b148c-d977-4e1c-9605-82731218c733"
      },
      "source": [
        "content = \"2010年底了，部队的友谊篮球赛就结束了\"\n",
        "seg_list = jieba.cut_for_search(content, HMM=False)\n",
        "\"/\".join(seg_list)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2010/年底/了/，/部队/的/友谊/篮球/球赛/篮球赛/就/结束/了'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0tHZLnL3GGBJ",
        "colab_type": "text"
      },
      "source": [
        "去除文本中的停用词"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KW3IvxYaGJI7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "70a4c675-9d10-4853-9a5b-c9ae94883642"
      },
      "source": [
        "content = \"2010年底了，部队的友谊篮球赛就结束了\"\n",
        "stop_words = set([\"了\", \"，\", \"的\", \"\", \"就\"])\n",
        "seg_list = jieba.cut(content)\n",
        "\"/\".join(filter(lambda x : x not in stop_words, seg_list))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2010/年底/部队/友谊/篮球赛/结束'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ie4dI99oJWrh",
        "colab_type": "text"
      },
      "source": [
        "添加专有名词"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mvw6Ul6CJVTu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "be38634f-0256-4223-cb5d-ab78908fc7ea"
      },
      "source": [
        "import jieba\n",
        "\n",
        "content = \"2010年底了，部队的友谊篮球赛就结束了\"\n",
        "stop_words = set([\"了\", \"，\", \"的\", \"\", \"就\"])\n",
        "jieba.add_word(\"友谊篮球赛\")\n",
        "seg_list = jieba.cut(content)\n",
        "\"/\".join(filter(lambda x : x not in stop_words, seg_list))"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2010/年底/部队/友谊篮球赛/结束'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2WsL589UoV9B",
        "colab_type": "text"
      },
      "source": [
        "### 3.4 自定义字典"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iaJX6P9zoiTe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}